Musical key classification

Machine Learning

WMAI010-05

Classifying keys in a piece of music using a Hidden Markov Model trained using Spotify data.

Let's visualize the Chroma vectors.

import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from IPython.display import HTML, Audio
import pandas as pd
import IPython
df = pd.read_csv('../example/39C5FuZ8C8M0QI8CrMsPkR.txt')
df

	start	duration	confidence	pitch	chroma
0	0.00000	0.11279	0.000	1.000	C (also B♯, D#)
1	0.00000	0.11279	0.000	0.707	C♯, D♭ (also B#)
2	0.00000	0.11279	0.000	0.374	D (also C#, E#)
3	0.00000	0.11279	0.000	0.391	D♯, E♭ (also F#)
4	0.00000	0.11279	0.000	0.390	E (also D#, F♭)
...	...	...	...	...	...
18199	466.14078	1.49923	0.382	0.672	G (also F#, A#)
18200	466.14078	1.49923	0.382	0.563	G♯, A♭
18201	466.14078	1.49923	0.382	0.490	A (also G#, B#)
18202	466.14078	1.49923	0.382	0.561	A♯, B♭ (also C#)
18203	466.14078	1.49923	0.382	0.445	B (also A#, C♭)

18204 rows × 5 columns

max_pitch = df.groupby('start')['pitch'].transform(max) == df['pitch']
linedf = df[max_pitch]
linedf

	start	duration	confidence	pitch	chroma
0	0.00000	0.11279	0.000	1.0	C (also B♯, D#)
13	0.11279	0.22390	1.000	1.0	C♯, D♭ (also B#)
25	0.33669	0.13324	0.459	1.0	C♯, D♭ (also B#)
39	0.46993	0.20295	0.484	1.0	D♯, E♭ (also F#)
55	0.67288	0.24562	0.729	1.0	G (also F#, A#)
...	...	...	...	...	...
18154	464.90414	0.19773	0.176	1.0	A♯, B♭ (also C#)
18156	465.10187	0.53959	0.479	1.0	C (also B♯, D#)
18173	465.64145	0.23755	0.373	1.0	F (also E♯, G#)
18185	465.87900	0.26177	0.114	1.0	F (also E♯, G#)
18194	466.14078	1.49923	0.382	1.0	D (also C#, E#)

1520 rows × 5 columns

def create_frame(frame, ax): # arguments are `frame number` and `fargs`
    ax.cla()
    window = 5 # show a window of 5 seconds
    step = frame/20 # advance 0.05 seconds per frame
    x1 = -window/2 + step
    x2 = window/2 + step
    scattersamples = (df['start'] >= x1) & (df['start'] <= x2)
    sns.scatterplot(x='start', y='chroma', size='pitch',
        data=df[scattersamples], ax=ax, hue='chroma', legend=False)
    linesamples = (linedf['start'] >= x1) & (linedf['start'] <= x2)
    sns.lineplot(x='start', y='chroma', alpha=0.5,
        data=linedf[linesamples], ax=ax, sort=False, legend=False)
    ax.set_xlim(x1, x2)
    line1, = ax.plot(
            [x1+window/2, x1+window/2],
            ['C (also B♯, D#)', 'B (also A#, C♭)'],
            '--', linewidth=2, label='Dashes set retroactively')
    plt.xlabel('time (in seconds)')
    plt.title('Chroma vectors of first 5 seconds of \'Foreplay / Long Time\' track.')
fig, ax = plt.subplots(figsize=(10, 5))
create_frame(0, ax)

fig, ax = plt.subplots(figsize=(10, 5))
animation = FuncAnimation(fig, create_frame, frames=300, fargs=(ax,),
    interval=50) # Interval at 1000/50 = 20 frames per second
HTML(animation.to_jshtml())

Once Loop Reflect

Audio("../example/39C5FuZ8C8M0QI8CrMsPkR.mp3")

animation.save('../example/39C5FuZ8C8M0QI8CrMsPkR.gif', writer='imagemagick', fps=20)

MovieWriter imagemagick unavailable; using Pillow instead.

Statistical analysis

Import packages

from tracklist import TrackList
from data import load_analysis, load_features
# import pandas as pd
from scipy import stats
import numpy as np
# import matplotlib.pyplot as plt
# import seaborn as sns
from scipy.stats import entropy
import matplotlib.pylab as pylab

---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-16-562d85327967> in <module>
      1 from tracklist import TrackList
----> 2 from data import load_analysis, load_features
      3 # import pandas as pd
      4 from scipy import stats
      5 import numpy as np

~/git/Key-Recognition/src/data.py in <module>
     13 from halo import Halo
     14 
---> 15 from args import get_args
     16 from constants import AUDIO_ANALYSIS, AUDIO_FEATURES
     17 from meta import Meta

~/git/Key-Recognition/src/args.py in <module>
      1 from argparse import ArgumentParser
      2 
----> 3 from constants import AUDIO_FEATURES, AUDIO_ANALYSIS
      4 
      5 

ImportError: cannot import name 'AUDIO_FEATURES' from 'scipy.optimize._highs.constants' (/Users/dunnkers/git/Key-Recognition/venv/lib/python3.8/site-packages/scipy/optimize/_highs/constants.cpython-38-darwin.so)

params = {'legend.fontsize': 'x-large',
          'figure.figsize': (15, 5),
         'axes.labelsize': 'x-large',
         'axes.titlesize':'x-large',
         'xtick.labelsize':'x-large',
         'ytick.labelsize':'x-large'}
pylab.rcParams.update(params)

Define variable

pitch_classes = {
    0: 'C (also B♯, D#)',
    1: 'C♯, D♭ (also B#)',
    2: 'D (also C#, E#)',
    3: 'D♯, E♭ (also F#)',
    4: 'E (also D#, F♭)',
    5: 'F (also E♯, G#)',
    6: 'F♯, G♭ (also E#)',
    7: 'G (also F#, A#)',
    8: 'G♯, A♭',
    9: 'A (also G#, B#)',
    10: 'A♯, B♭ (also C#)',
    11: 'B (also A#, C♭)'
}
data_dir = '../dataset'
song = 0
trim_length = 20

Load data

meta = TrackList.load_from_dir(data_dir)
all_tracks = meta.get_track_ids()

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-10-98af7b1efd00> in <module>
----> 1 meta = TrackList.load_from_dir(data_dir)
      2 all_tracks = meta.get_track_ids()

NameError: name 'TrackList' is not defined

Investigate a single song

track_id = meta.get_track_ids()[song]
analysis = load_analysis(data_dir, track_id)
duration = analysis['duration']
key = pitch_classes[analysis['key']]
print(f"Track {track_id} lasts {duration} seconds and is in the {key} key")
analysis['segments'] = [seg for seg in analysis["segments"] if seg["start"] < trim_length]
segments = analysis['segments']
print(f"And has {len(segments)} segments in the first {trim_length} seconds")

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-12-90536a60de34> in <module>
----> 1 track_id = meta.get_track_ids()[song]
      2 analysis = load_analysis(data_dir, track_id)
      3 duration = analysis['duration']
      4 key = pitch_classes[analysis['key']]
      5 print(f"Track {track_id} lasts {duration} seconds and is in the {key} key")

NameError: name 'meta' is not defined

track_df = pd.DataFrame(segments)
# Map value of pitches to keys and get max pitch
track_df["pitches"] = [dict(zip(pitch_classes.values(), track_df.pitches[i])) for i in range(len(track_df))]
track_df["chroma"] = [max(track_df.pitches[i], key=track_df.pitches[i].get) for i in range(len(track_df))]
track_df["max_pitch"] = [max(track_df.pitches[0].values()) for i in range(len(track_df))]

track_df.head()

	start	duration	confidence	pitches	chroma	max_pitch
0	0.00000	0.06998	0.000	{'C (also B♯, D#)': 1.0, 'C♯, D♭ (also B#)': 0...	C (also B♯, D#)	1.0
1	0.06998	0.16118	0.325	{'C (also B♯, D#)': 0.949, 'C♯, D♭ (also B#)':...	A♯, B♭ (also C#)	1.0
2	0.23116	0.24621	0.577	{'C (also B♯, D#)': 1.0, 'C♯, D♭ (also B#)': 0...	C (also B♯, D#)	1.0
3	0.47737	0.27166	0.906	{'C (also B♯, D#)': 0.224, 'C♯, D♭ (also B#)':...	G♯, A♭	1.0
4	0.74902	0.52299	1.000	{'C (also B♯, D#)': 1.0, 'C♯, D♭ (also B#)': 0...	C (also B♯, D#)	1.0

df = pd.concat([pd.DataFrame(segment, pitch_classes.values()).reset_index() for segment in segments])
df.columns = ['chroma', 'start', 'duration', 'confidence', 'max_pitch']

plt.plot(track_df.start, track_df.chroma)
plt.scatter(df.start, df.chroma, s=df.max_pitch, cmap=df.chroma)
plt.xlabel('time (in seconds)')
plt.title(f'Chroma vectors of first {trim_length} seconds of {track_id} track.')
plt.show()

Investigate overall dataset

analyses = [load_analysis(data_dir, track) for track in all_tracks]
# analyses_trimmed = [load_analysis(data_dir, track) for track in all_tracks]

df = pd.DataFrame([analyses[i]["key"] for i in range(len(all_tracks))], columns=["pitch"])
df['key_confidences'] = [analyses[i]["key_confidence"] for i in range(len(all_tracks))]
df['modes'] = [analyses[i]["mode"] for i in range(len(all_tracks))]
df['mode_confidences'] = [analyses[i]["mode_confidence"] for i in range(len(all_tracks))]
df['durations'] = [analyses[i]["duration"] for i in range(len(all_tracks))]
df['no_segments'] = [len(analyses[i]["segments"]) for i in range(len(all_tracks))]
df['segment_lens'] = [[np.mean([seg['duration'] for seg in analyses[i]["segments"]])][0] for i in range(len(all_tracks))]
df['entropies'] = [np.mean(entropy([seg['pitches'] for seg in track['segments']], base=12)) for track in analyses]
for track in range(len(all_tracks)):
    analyses[track]['segments'] = [seg for seg in analyses[track]["segments"] if seg["start"] < trim_length]

df['no_segments_trimmed'] = [len(analyses[i]["segments"]) for i in range(len(all_tracks))]
df['segment_lens_trimmed'] = [[np.mean([seg['duration'] for seg in analyses[i]["segments"]])][0] for i in range(len(all_tracks))]
df['entropies_trimmed'] = [np.mean(entropy([seg['pitches'] for seg in track['segments']], base=12)) for track in analyses]
df.describe()

	pitch	key_confidences	modes	mode_confidences	durations	no_segments	segment_lens	entropies	no_segments_trimmed	segment_lens_trimmed	entropies_trimmed
count	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000	1004.000000
mean	5.508964	0.472093	0.501992	0.490950	234.387876	888.515936	0.269700	2.575403	75.120518	0.284871	1.571006
std	3.457608	0.253061	0.500245	0.183925	71.751231	301.531215	0.043916	0.124502	16.654917	0.085877	0.127432
min	0.000000	0.000000	0.000000	0.000000	78.200000	271.000000	0.169184	2.075766	14.000000	0.127709	0.659707
25%	2.000000	0.285750	0.000000	0.375000	193.937770	708.000000	0.237530	2.496891	65.000000	0.239472	1.502508
50%	6.000000	0.486000	1.000000	0.504500	221.840000	826.500000	0.263967	2.565132	74.000000	0.271023	1.582820
75%	9.000000	0.664000	1.000000	0.616000	261.084705	995.000000	0.296643	2.642356	84.000000	0.312653	1.657781
max	11.000000	1.000000	1.000000	1.000000	1352.413300	4284.000000	0.479609	3.235441	157.000000	1.499331	1.842023

mode_intervals = [len(df[df.mode_confidences >= i])/len(df) for i in np.arange(0, 1, 0.1)]
plt.plot(np.arange(0, 1, 0.1), mode_intervals, label="Mode Confidence", color="red")
key_intervals = [len(df[df.key_confidences >= i])/len(df) for i in np.arange(0, 1, 0.1)]
plt.plot(np.arange(0, 1, 0.1), key_intervals, label="Key Confidence", color="blue")
plt.xlabel('Confidence Lower Bound')
plt.ylabel('Fraction of tracks')
plt.title(f'The fraction of tracks within a given mode and key confidence')
for i, v in zip(np.arange(0, 1, 0.1), mode_intervals):
    plt.text(i, v+0.01, str(int(v*100))+"%", fontsize=12, color="red")
for i, v in zip(np.arange(0, 1, 0.1), key_intervals):
    plt.text(i, v-0.1, str(int(v*100))+"%", fontsize=12, color="blue")
plt.legend()
plt.show()

Double check that we have selected an equal amount of each key and mode

df_comb = df.replace({"pitch": pitch_classes}).replace({"modes": {0: "Minor", 1: "Major"}})[["pitch", "modes"]]
df_comb.groupby(['pitch', 'modes']).size().unstack().plot(kind='bar', stacked=True, alpha=0.5)
plt.xticks(rotation=45)
plt.xlabel('Keys')
plt.ylabel('Number of tracks')
plt.title(f'The number of tracks with a given mode and key')
plt.show()

plt.hist(df.durations, alpha=0.5)
plt.xticks(rotation=45)
plt.xlabel('Time (seconds)')
plt.ylabel('Number of tracks')
plt.title(f'The number of tracks within a given song length')
plt.show()

plt.hist(df.no_segments, alpha=0.5)
plt.xticks(rotation=45)
plt.xlabel('Number of segments')
plt.ylabel('Number of tracks')
plt.title(f'The number of tracks within a given number of segments')
plt.show()

Compare Trimmed songs to not trimmed

plt.hist([df.entropies, df.entropies_trimmed], alpha=0.5, label=["Full song", "%s s trimmed song"%trim_length])
plt.xticks(rotation=45)
plt.xlabel('Average segment entropy')
plt.ylabel('Number of tracks')
plt.title(f'The number of tracks within a given average segment entropy')
plt.legend()
plt.show()

plt.hist([df.segment_lens, df.segment_lens_trimmed], alpha=0.5, label=["Full song", "%s s trimmed song"%trim_length])
plt.xticks(rotation=45)
plt.xlabel('Average segment length')
plt.ylabel('Number of tracks')
plt.title(f'The number of tracks within a given average segment length')
plt.show()

x = 3+1j*4
abs(x)**2

25.0

About

Built by Jeroen Overschie, Jordan Barkai, Thijs Havinga and Pieter Dekker - as part of the Machine Learning course WMAI010-05 taught on the University of Groningen.